# -*- coding: utf-8 -*-
import json
import random
import scrapy
from MeiHaoTong.user_agent import Android
from MeiHaoTong.wxb import lis_cate,GenerateCookie


class WeiXiaoBaoSpider(scrapy.Spider):
    name = 'weixiaobao'
    gc = GenerateCookie()
    custom_settings = {
        "DOWNLOAD_DELAY":8,
    }

    def start_requests(self):
        for dic in lis_cate:
            for key,value_list in dic.items():
                for fakeid,source in value_list:
                    headers = {
                        'Cookie':'visit-wxb-id=; PHPSESSID={}'.format(random.choice(self.gc.get_cookie())),
                        'Referer':'https://data.wxb.com/details/postRead?id={}'.format(fakeid),
                        'User-Agent':random.choice(Android),
                        'Host': 'data.wxb.com',
                        'X-Requested-With':'XMLHttpRequest'
                    }
                    url = 'https://data.wxb.com/account/statArticles/{}?period=30&page=1&sort='.format(fakeid)
                    yield scrapy.Request(url,headers=headers,callback=self.parse,meta={'cate':key,'source':source})


    def parse(self, response):
        res = json.loads(response.text).get('data')
        for info in res:
            item = {}
            item['title'] = info.get('title')
            item['url'] = info.get('url')
            item['publish_time'] = info.get('push_date')+':00'
            item['author'] = response.meta['source']
            item['site_classify_id'] = response.meta['cate']
            item['comment_count'] = info.get('like_num')
            item['site_id'] = 105 #3表示微信
            brose = info.get('read_num') #浏览量
            if '万+' in str(brose):
                brose = int(brose.split('万+')[0])*10000
            item['browse_count'] = brose
            yield item
            # print(item)
